package com.chance.cc.crawler.development.bootstrap.xiaohongshu;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.queue.crawler.CrawlerQueueConfig;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.lang3.StringUtils;

import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
import java.util.Map;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;
import static org.codehaus.groovy.runtime.EncodingGroovyMethods.md5;

public class XHSUserNodeStart {

    private static final String domainId = "xhs";
    public static final String site= "user_notes";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) throws NoSuchAlgorithmException {
//        userNotes();
        userNotesOne();

    }

    private static void userNotes(){
        String url  = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/user/";
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpUrl(url)
                .releaseTime(System.currentTimeMillis())
                .needWashed(false)
                .needParsed(false)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null))
                .proxy(proxy)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addDomain(domainId);
        requestRecord.tagsCreator().bizTags().addSite(site);


        CrawlerRequestRecord userOauthInfoCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("xhs_user_note_oauth_infos", turnPageItem)
                .httpUrl("http://192.168.1.215:9599/crawler/oauth/api/v1/xhs/userOauthInfos")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        CrawlerRequestRecord userKeyWordCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("xhs_user_note_kw", turnPageItem)
                .httpUrl("http://192.168.1.215:9599/v1/meta/xhs/keys?site=user_ids")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        CrawlerQueueConfig crawlerQueueConfig = DevCrawlerController.devRequestQueue(StringUtils.joinWith("-",  domainId,"user","notes", "queue"));
        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .triggerInfo(domainId, "cron", System.currentTimeMillis(), "realtime")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(StringUtils.joinWith("-", "crawler", domainId, "queue"))) //内存队列
                .consoleResultPipeline("console") //控制台输t出
                .fileResultPipeline("file", "D:\\chance\\data\\xhs\\xhs-user-notes.json", false)
                .requestRecord(requestRecord)
                .supportRecord(userOauthInfoCrawlerRecord)
                .supportRecord(userKeyWordCrawlerRecord)
                .crawlerThreadNum(1)
                .build();
        //添加评论去重信息
        CrawlerRecord filterCrawlerRecord = new CrawlerRecord();
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.dateRange);
        filterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-",filter,domainId,site,"comment")));
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null));
        requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addLabelTag(CrawlerEnum.CrawlerDataType.comment.enumVal());
        devCrawlerController.start();
    }

    private static void userNotesOne(){
        String url  = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/user/52d8c541b4c4d60e6c867480/notes?page=1&page_size=15";
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpUrl(url)
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7,null))
                .needWashed(true)
                .proxy(proxy)
                .build();
        requestRecord.tagsCreator().bizTags().addDomain(domainId);
        requestRecord.tagsCreator().bizTags().addSite(site);


        Map<String,String> headers = new HashMap<>();
        String x_sign = null;
        try {
            x_sign = "X" + md5("/fe_api/burdock/weixin/v2/user/52d8c541b4c4d60e6c867480/notes?page=1&page_size=15" + "WSUDD");
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        }
        headers.put("charset", "utf-8");
        headers.put("Accept-Encoding", "gzip,compress,br,deflate");
        headers.put("referer", "https://servicewechat.com/wxb296433268a1c654/37/page-frame.html");
        headers.put("device-fingerprint","WHJMrwNw1k/H7wa/KU0af7xERBoHA6AK3QmMCeRPoEhH2f80ejmAyKX8ZWiuMoy8cmaHxFZbKWza+FoOeeyiKah0RVKRYVzoUdCW1tldyDzmauSxIJm5Txg==1487582755342");
        headers.put("authorization", "wxmp.86ef82bc-09c1-4f2b-9bf4-8d0bf5f670d8");
        headers.put("content-type", "application/json");
        headers.put("User-Agent", "Mozilla/5.0 (Linux; Android 6.0; DIG-AL00 Build/HUAWEIDIG-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Mobile Safari/537.36 MicroMessenger/7.0.3.1400(0x27000334) Process/appbrand0 NetType/WIFI Language/zh_CN");
        headers.put("Host", "www.xiaohongshu.com");
        headers.put("Connection", "Keep-Alive");
        headers.put("x-sign",x_sign);
        requestRecord.getHttpRequest().setHeaders(headers);
//        requestRecord.getHttpConfig().setResponseTextGenerateHtml(false);

        CrawlerRequestRecord userOauthInfoCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("xhs_user_note_oauth_infos", turnPageItem)
                .httpUrl("http://192.168.1.215:9599/crawler/oauth/api/v1/xhs/userOauthInfos")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        CrawlerQueueConfig crawlerQueueConfig = DevCrawlerController.devRequestQueue(StringUtils.joinWith("-",  domainId,"user","notes", "queue"));
        DevCrawlerController.builder()
                .triggerInfo(domainId,"cron",System.currentTimeMillis(),"realtime")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(StringUtils.joinWith("-","crawler",domainId,"queue"))) //内存队列
                .consoleResultPipeline() //控制台输t出
                .queueResultPipeline("article_ids",crawlerQueueConfig)
                .requestRecord(requestRecord)
//                .supportRecord(userOauthInfoCrawlerRecord)
                .build()
                .start();
    }
}
